'use strict';

const { MongoClient } = require('mongodb');

class MongoPipeline {

    open_spider(spider) {
        const server = spider.options.ds;
        if (!server) {
            spider.log.error('You must set Mongodb connect str first!')
            process.exit(1)
        }
        this.client = new MongoClient(server);
    }

    async process_item(item, spider, request) {
        const { log } = spider;
        const { target } = spider.options;
        const { index_name, id, file = {}, ...body } = item;
        const rowId = id ? id : file.basename || request.reqKey;

        if (target && rowId) {
            await this.client.connect();
            const collection = this.client.db().collection(target);
            try {
                await collection.deleteOne({
                    _id: spider.name + ":" + rowId,
                })
                await collection.insertOne({
                    _id: spider.name + ":" + rowId,
                    _spider_name: spider.name,
                    _spider_time: new Date(),
                    _spider_uri: request.link,
                    ...file,
                    ...body
                })
            } catch (error) {
                console.log('insert error: ', error)
            }
            log.debug(`${target} index ${rowId} ok`)
            this.client.close();
        }
        return item;
    }

}

module.exports = MongoPipeline;